from urllib.request import urlopen
from urllib.parse import urlparse
from bs4 import BeautifulSoup
import re
import datetime
import random

pages=set()
random.seed(datetime.datetime.now)


def getInteralLinks(bsObj,includeUrl):
    includeUrl=urlparse(includeUrl).scheme+"://"+urlparse(includeUrl).netloc
    interalLiks=[]
    for link in bsObj.findAll("a",hraf=re.compile("^(/|.*"+includeUrl+")")):
        if link.attrs['href'] not in interalLiks:
            interalLiks.append(link.attrs['href'])
        else:
            interalLiks.append(link.attrs['href'])
    return interalLiks

def getExteenalLinks(bsObj, excludeUrl):
    excludeList =[]
    for link in bsObj.findAll("a",href=re.compile("^(http|www)((?!"+excludeUrl+").)*$")):
        if link.attrs['href'] is not None:
            if link.attrs['href'] not in excludeList:
                excludeList.append(link.attrs['href'])
    return excludeList

def spltAddress(address):
    addressParts=address.replace("http://","").split("/")
    return addressParts

def getRandomExternList(startingPage):
    html=urlopen(startingPage)
    bsObj=BeautifulSoup(html)
    externlList=getExteenalLinks(bsObj,spltAddress(startingPage)[0])
    if len(externlList)==0:
        print("Np external links aroud:")
        domain=urlparse(startingPage).scheme+"://"+urlparse(startingPage).netloc
        intternalLinks=getExteenalLinks(bsObj,domain)
        intternalLinks=getInteralLinks(startingPage)
        return getNextExternalLink(intternalLinks[readom.randint(0,len(intternalLinks)-1)])

    else:
        return externlList[random.randint(0,len(externlList)-1)]

def followExternalOnly (startingSite):
    externalLink=getRandomExternList(startingSite)
    print("Random external link is :"+externalLink)
    followExternalOnly(externalLink)

    
followExternalOnly("http://www.quanshuwang.com/")
